import requests
import re


class Begin:

    def __init__(self, cookies, header=None):
        '传入cookies，header'
        self.origin_url = r'http://localhost:8080/itm.supplier.client/supplier/manage/sy/shouye.itm?id=11'
        self.header = header
        self.cookies = cookies

    def get_res(self):
        '访问起始网址所有的链接'
        res = requests.get(self.origin_url, cookies=self.cookies)
        rule = re.compile(r'href=\"(.+?)\"')
        urls = rule.findall(res.text)
        return urls

    def url_Screening(self, urls):
        '数据清洗，特定结尾的数据清除掉'
        urls_res = []
        # css、js、png、jpg、bmp结尾的网址清除
        css_end = re.compile('css$')
        js_end = re.compile('js$')
        png_end = re.compile('png$')
        jpg_end = re.compile('jpg$')
        bmp_end = re.compile('bmp$')
        logout = re.compile('logout')
        javascript = re.compile('javascript')
        screening = [css_end, js_end, png_end, jpg_end, bmp_end, logout,javascript]
        for i in urls:
            flag = 0
            for j in screening:
                if j.search(i):
                    flag = 1
            if flag == 0:
                urls_res.append(i)
        return urls_res

    def url_ergodic(self, numbers):
        '遍历url，设定遍历网站的个数'
        count = 0
        ergodic_urls = [self.origin_url]
        while count<numbers:
            # 访问列表中的网址
            try:
                res = requests.get(ergodic_urls[count], cookies=self.cookies, params={'keepalive':'False'})
            except Exception as e:
                print(e)
            rule = re.compile(r'href=\"(.+?)\"')
            urls = rule.findall(res.text)
            # 得到清洗之后的网址
            urls_res = self.url_Screening(urls)
            # 放入循环，如果不是重复的，加入到遍历的url中
            for i in urls_res:
                if i in ergodic_urls:
                    continue
                else:
                    ergodic_urls.append(i)
            count += 1
        return ergodic_urls


cookies = {'sid':'b798b030-e018-49c5-bcff-6c952bc41ee6'}
t = Begin(cookies=cookies)
t.url_ergodic(20)